def plot_with_corners(image, corners_matrix, inline: bool=True):
import matplotlib.pyplot as plt
if inline:
# show inline image for the reader
%matplotlib inline
else:
# show in external window to manually read the coordinates
%matplotlib qt
plt.figure(figsize=(20,10))
plt.imshow(image)
plt.plot(corners_matrix[0][0], corners_matrix[0][1], "Xr") # top-left red star
plt.plot(corners_matrix[1][0], corners_matrix[1][1], "Xb") # top-right red star
plt.plot(corners_matrix[2][0], corners_matrix[2][1], "Xg") # bottom-right red star
plt.plot(corners_matrix[3][0], corners_matrix[3][1], "Xy") # bottom-left red star
by Uki D. Lucas
This project is written to meet following requirements: https://review.udacity.com/#!/rubrics/571/view
The goals / steps of this project are the following:
The images for camera calibration are stored in the folder called camera_cal.
The images in test_images are for testing your pipeline on single frames.
If you want to extract more test images from the videos, you can simply use an image writing method like cv2.imwrite(), i.e., you can read the video in frame by frame as usual, and for frames you want to save for later you can write to an image file.
To help the reviewer examine your work, please save examples of the output from each stage of your pipeline in the folder called ouput_images, and include a description in your writeup for the project of what each image shows.
The video called project_video.mp4 is the video your pipeline should work well on.
The challenge_video.mp4 video is an extra (and optional) challenge for you if you want to test your pipeline under somewhat trickier conditions.
The harder_challenge.mp4 video is another optional challenge and is brutal!
If you're feeling ambitious (again, totally optional though), don't stop there!
We encourage you to go out and take video of your own, calibrate your camera and show us how you would implement this project from scratch!
The very well documented code for this step is contained in document camera_calibration available in HTML, ipynb and py formats.
This image was generated by Uki D. Lucas

def __get_sample_gray(image_file_name: str):
import cv2 # we will use OpenCV library
image_original = cv2.imread(image_file_name)
# convert BGR image to gray-scale
image_gray = cv2.cvtColor(image_original, cv2.COLOR_BGR2GRAY)
return image_original, image_gray
def plot_images(left_image, right_image):
import numpy as np
import matplotlib.pyplot as plt
plt.figure(figsize=(20,10))
plot_image = np.concatenate((left_image, right_image), axis=1)
plt.imshow(plot_image)
plt.show()
def prep_calibration(image_file_names: list, use_optimized = True, verbose = False):
# we will use OpenCV library
import cv2
# find CORNERS
object_point_list, image_points_list = __find_inside_corners(image_file_names)
# get sample image, mostly for dimensions
image_original, image_gray = __get_sample_gray(image_file_names[1])
# Learn calibration
# Returns:
# - camera matrix
# - distortion coefficients
# - rotation vectors
# - translation vectors
has_sucess, matrix, distortion, rvecs, tvecs = cv2.calibrateCamera(
object_point_list,
image_points_list,
image_gray.shape[::-1],
None,
None)
## I can use this to improve the calibration (no cropped edges, but curved edges)
image_dimentions = image_original.shape[:2] # height, width
matrix_optimized, roi = cv2.getOptimalNewCameraMatrix(
matrix,
distortion,
image_dimentions,
1,
image_dimentions)
return matrix, matrix_optimized, distortion
import glob
image_file_names = glob.glob("camera_cal/calibration*.jpg")
print(len(image_file_names), "images found")
def __find_inside_corners(image_file_names: list, nx: int=9, ny: int=6, verbose = False):
"""
Chessboard dimentsions:
nx = 9 # horizontal
ny = 6 # vertical
"""
import cv2 # we will use OpenCV library
import numpy as np
# Initialise arrays
# Object Points: points on the original picture of chessboard
object_point_list = []
#Image Points: points on the perfect 2D chessboard
image_points_list = []
# Generate 3D object points
object_points = np.zeros((nx*ny, 3), np.float32)
object_points[:,:2] = np.mgrid[0:nx, 0:ny].T.reshape(-1, 2)
#print("first 5 elements:\n", object_points[0:5])
# see: http://docs.opencv.org/trunk/dc/dbb/tutorial_py_calibration.html
termination_criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
chessboard_dimentions = (nx, ny)
import matplotlib.pyplot as plt
for image_file_name in image_file_names:
if verbose:
print("processing image:", image_file_name)
image_original, image_gray = __get_sample_gray(image_file_name)
# Find the chess board corners
# Paramters:
# - image_gray
# - the chessboard to be used is 9x6
# - flags = None
has_found, corners = cv2.findChessboardCorners(image_gray, chessboard_dimentions, None)
if has_found == True:
# fill in ObjectPoints
object_point_list.append(object_points)
corners2 = cv2.cornerSubPix(image_gray, corners, (11,11), (-1,-1), termination_criteria)
# fill in ImagePoints
image_points_list.append(corners2)
# Draw and display the corners
# I have to clone/copy the image because cv2.drawChessboardCorners changes the content
image_corners = cv2.drawChessboardCorners(
image_original.copy(),
chessboard_dimentions,
corners2,
has_found)
if verbose:
plot_images(image_original, image_corners)
else: # not has_found
if verbose:
print("The", chessboard_dimentions,
"chessboard pattern was not found, most likely partial chessboard showing")
plt.figure()
plt.imshow(image_original)
plt.show()
# end if has_found
# end for
return object_point_list, image_points_list
#import camera_calibration as cam # local camera_calibration.py, same directory
camera_matrix, matrix_optimized, distortion_coefficients = prep_calibration(
image_file_names,
use_optimized = True)
#image_file_path = "test_images/test1.jpg"
#image_file_path = "test_images/stop_sign_angle_001.png"
image_file_path = "camera_cal/calibration8.jpg"
import os
import matplotlib.image as mpimg
if os.path.isfile(image_file_path):
image = mpimg.imread(image_file_path)
import matplotlib.pyplot as plt
# show in external window to manually read the coordinates
#%matplotlib qt
# show inline image for the reader
#%matplotlib inline
#plt.imshow(image)
import cv2 # we will use OpenCV library
image = cv2.imread(image_file_path)
image_corrected1 = cv2.undistort(image, camera_matrix, distortion_coefficients, None, None)
plot_images(image, image_corrected1)
image_corrected2 = cv2.undistort(image, camera_matrix, distortion_coefficients, None, matrix_optimized)
plot_images(image, image_corrected2)
I used a combination of color and gradient thresholds to generate a binary image (thresholding steps at lines # through # in another_file.py). Here's an example of my output for this step. (note: this is not actually from one of the test images)
# show in external window to manually read the coordinates
#%matplotlib qt
# show inline image for the reader
%matplotlib inline
#plt.imshow(image)plot_images(image, warped)
import numpy as np
# calibration8 source
SRC = np.float32([
[657,115],
[1021,175],
[1021,536],
[665,613]])
plot_with_corners(image_corrected1, SRC, inline=True)
import numpy as np
# calibration8 destination
DEST = np.float32([
[110,100],
[1180,100],
[1180,620],
[100,620]])
plot_with_corners(image_corrected1, DEST, inline=True)
# http://docs.opencv.org/3.1.0/da/d6e/tutorial_py_geometric_transformations.html
M = cv2.getPerspectiveTransform(SRC, DEST)
print(M)
img_size = (image_corrected1.shape[1], image_corrected1.shape[0])
image_warped = cv2.warpPerspective(image_corrected1, M, img_size)
plot_with_corners(image_warped, DEST, inline=True)
image_file_path = "test_images/test4.jpg"
import cv2 # we will use OpenCV library
image = cv2.imread(image_file_path)
image_corrected = cv2.undistort(image, camera_matrix, distortion_coefficients, None, None)
plot_images(image, image_corrected)
image_file_path = "test_images/straight_lines1.jpg"
import cv2 # we will use OpenCV library
image = cv2.imread(image_file_path)
import numpy as np
img_size = (image.shape[1], image.shape[0])
width = image.shape[1]
height = image.shape[0]
horizon_offset = 90
top_horizontal_offset = 55
car_hood_offset = 45
hood_to_width_offset = 200
SRC = np.float32(
[[(width / 2) - top_horizontal_offset, height / 2 + horizon_offset], #619, 432
[hood_to_width_offset, height - car_hood_offset],
[width - hood_to_width_offset, height - car_hood_offset],
[(width / 2 + top_horizontal_offset), height / 2 + horizon_offset]])
print(SRC)
DEST = np.float32(
[[(width / 4), 0],
[(width / 4), img_size[1]],
[(width * 3 / 4), img_size[1]],
[(width * 3 / 4), 0]])
print(DEST)
plot_with_corners(image, SRC, inline=True)
I am using the reference image with staight lines to determine this camera's horizon (areas to mask)
plot_with_corners(image_corrected, SRC, inline=True)
# http://docs.opencv.org/3.1.0/da/d6e/tutorial_py_geometric_transformations.html
M = cv2.getPerspectiveTransform(SRC, DEST)
print(M)
image_warped = cv2.warpPerspective(image_corrected, M, img_size)
#image_warped.reshape(image_warped.shape[0], image_warped.shape[1])
plot_with_corners(image_warped, DEST, inline=True)
def mask_lane_lines(img):
'''
Method masks lane lines.
'''
img = np.copy(img)
#Blur
kernel = np.ones((5,5),np.float32)/25
img = cv2.filter2D(img,-1,kernel)
#YUV for histogram equalization
yuv = cv2.cvtColor(img, cv2.COLOR_BGR2YUV)
yuv[:,:,0] = cv2.equalizeHist(yuv[:,:,0])
img_wht = cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR)
#Compute white mask
img_wht=img_wht[:,:,1]
img_wht[img_wht<250]=0
mask_wht = cv2.inRange(img_wht, 250, 255)
yuv[:,:,0:1]=0
#Yellow mask
kernel = np.ones((5,5),np.float32)/25
dst = cv2.filter2D(yuv,-1,kernel)
sobelx = np.absolute(cv2.Sobel(yuv[:,:,2], cv2.CV_64F, 1, 0,ksize=5))
sobelx[sobelx<200]=0
sobelx[sobelx>=200]=255
#Merge mask results
mask = mask_wht + sobelx
return mask
mask = mask_lane_lines(img = image_warped)
print(mask)
After the lens distortion has been removed, a binary image will be created, containing pixels which are likely part of a
lane. Therefore the result of multiple techniques are combined by a bitwise and operator. Finding good parameters for the different techniques like threshold values is quite challenging. To improve the feedback cycle of applying different parameters, an interactive jupyter notebook was created.
The first technique is called sobel operation which is able to detect edges by computing an approximation of the
gradient of the image intensity function. The operation was applied for both directions (x and y) and combined to keep only
those pixel which are on both results and also over a specified threshold. An averaged gray scale image from
the U and V color channels of the YUV space and also the S channel of the HLS space was used as input.
Additionally, the magnitude and direction of the gradient was calculated and combined by keeping only pixels with values above a threshold (different threshold for magnitude and direction) on both images.
Technique number three is a basic color thresholding which tries to isolate yellow pixels.
The last technique is an adaptive highlight / high intensity detection. It isolates all the pixels which have values above
a given percentile in order to make it more resilient against different lighting conditions.
In the end, the results are combined through a bitwise or operation to get the final lane mask.
| Sobel X & Y | Magnitude & Direction of Gradient | Yellow | Highlights | Combined |
|---|---|---|---|---|
![]() |
![]() |
![]() |
![]() |
![]() |
To determine suitable source coordinates for the perspective transformation, an image with relative straight lines was used as reference. Since the car was not perfectly centered the image, it was horizontally mirrored. The resulting image was then used inside the interactive jupyter notebook to fit vanishing lines and to get source coordinates.
After suitable coordinates were determined, the transformation can be applied to other images. This is of course just an approximation and is not 100% accurate.
| Mask | Birdseye View |
|---|---|
![]() |
![]() |
Since not all pixels marked in the mask are actually part of the lanes, the most likely ones have to be identified. For that, a sliding histogram is applied to detect clusters of marked pixels. The highest peak of each histogram is used as the center of a window which assigns each pixel inside to the corresponding lane. The sliding histogram is applied to the left half of the image to detect left line pixels and applied on the right half of the image to detect right lane pixels. Therefore, the algorithm will fail if a lane crosses the center of the image.
This process is pretty computing intensive. That's why the algorithm will try to find lane pixels in the area of
previously found lines first. This is only possible when using videos.
| Left Lane Histogram | Assigned Pixels | Right Lane Histogram |
|---|---|---|
![]() |
|
Then I did some other stuff and fit my lane lines with a 2nd order polynomial kinda like this:
Fit Visual

I did this in lines # through # in my code in my_other_file.py
I implemented this step in lines # through # in my code in yet_another_file.py in the function map_lane(). Here is an example of my result on a test image:
Binary Example

Warp Example

With pixels assigned to each lane, a second order polynomials can be fitted. To achieve smoother results, the polynomials are also average over the last five frames in a video. The polynomials are also used to calculate the curvature of the lane and the relative offset from the car to the center line.
| Fit Polynomial | Final |
|---|---|
![]() |
![]() |
Here's a link to my video result
Output

Video
Here I'll talk about the approach I took, what techniques I used, what worked and why, where the pipeline might fail and how I might improve it if I were going to pursue this project further.
# see http://nbconvert.readthedocs.io/en/latest/usage.html
!jupyter nbconvert --to markdown README.ipynb